From f1cd150e1f99263722fe840b4e039ec8e14dca48 Mon Sep 17 00:00:00 2001 From: "cl349@freefall.cl.cam.ac.uk" Date: Fri, 27 Aug 2004 13:29:44 +0000 Subject: [PATCH] bitkeeper revision 1.1159.57.1 (412f374873CWRyXQT5wTVtsSUl961g) Use hypercall batching where possible. --- .../arch/xen/i386/kernel/ldt.c | 5 +- .../arch/xen/i386/kernel/traps.c | 1 + .../arch/xen/i386/mm/hypervisor.c | 128 +++++++++++++++++- .../arch/xen/i386/mm/init.c | 8 +- .../arch/xen/i386/mm/pgtable.c | 6 +- .../drivers/xen/blkfront/blkfront.c | 2 +- .../include/asm-xen/asm-i386/mmu_context.h | 4 +- .../include/asm-xen/asm-i386/pgalloc.h | 2 + .../include/asm-xen/asm-i386/pgtable-2level.h | 6 +- .../include/asm-xen/asm-i386/pgtable.h | 21 ++- .../include/asm-xen/asm-i386/processor.h | 6 - .../include/asm-xen/asm-i386/tlbflush.h | 9 +- .../include/asm-xen/hypervisor.h | 11 ++ 13 files changed, 172 insertions(+), 37 deletions(-) diff --git a/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/ldt.c b/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/ldt.c index 9899654926..4dd697384c 100644 --- a/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/ldt.c +++ b/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/ldt.c @@ -22,8 +22,10 @@ #ifdef CONFIG_SMP /* avoids "defined but not used" warnig */ static void flush_ldt(void *null) { - if (current->active_mm) + if (current->active_mm) { load_LDT(¤t->active_mm->context); + flush_page_update_queue(); + } } #endif @@ -90,6 +92,7 @@ static inline int copy_ldt(mm_context_t *new, mm_context_t *old) memcpy(new->ldt, old->ldt, old->size*LDT_ENTRY_SIZE); make_pages_readonly(new->ldt, (new->size * LDT_ENTRY_SIZE) / PAGE_SIZE); + flush_page_update_queue(); return 0; } diff --git a/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/traps.c b/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/traps.c index 9cef6bd77e..2f3529b759 100644 --- a/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/traps.c +++ b/linux-2.6.8.1-xen-sparse/arch/xen/i386/kernel/traps.c @@ -967,6 +967,7 @@ void __init trap_init(void) set_call_gate(&default_ldt[0],lcall7); set_call_gate(&default_ldt[4],lcall27); __make_page_readonly(&default_ldt[0]); + xen_flush_page_update_queue(); /* * Should be a barrier for any external CPU state. diff --git a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/hypervisor.c b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/hypervisor.c index 957555f92a..a57eabcd8c 100644 --- a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/hypervisor.c +++ b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/hypervisor.c @@ -28,7 +28,7 @@ static spinlock_t update_lock = SPIN_LOCK_UNLOCKED; #define QUEUE_SIZE 2048 #define pte_offset_kernel pte_offset #else -#define QUEUE_SIZE 1 +#define QUEUE_SIZE 128 #endif static mmu_update_t update_queue[QUEUE_SIZE]; @@ -152,6 +152,12 @@ static inline void increment_index(void) if ( unlikely(idx == QUEUE_SIZE) ) __flush_page_update_queue(); } +static inline void increment_index_and_flush(void) +{ + idx++; + __flush_page_update_queue(); +} + void queue_l1_entry_update(pte_t *ptr, unsigned long val) { unsigned long flags; @@ -271,6 +277,126 @@ void queue_machphys_update(unsigned long mfn, unsigned long pfn) spin_unlock_irqrestore(&update_lock, flags); } +/* queue and flush versions of the above */ +void xen_l1_entry_update(pte_t *ptr, unsigned long val) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); +#if MMU_UPDATE_DEBUG > 3 + DEBUG_disallow_pt_read((unsigned long)ptr); +#endif + update_queue[idx].ptr = virt_to_machine(ptr); + update_queue[idx].val = val; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_l2_entry_update(pmd_t *ptr, unsigned long val) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = virt_to_machine(ptr); + update_queue[idx].val = val; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_pt_switch(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = phys_to_machine(ptr); + update_queue[idx].ptr |= MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_NEW_BASEPTR; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_tlb_flush(void) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_TLB_FLUSH; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_invlpg(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = MMU_EXTENDED_COMMAND; + update_queue[idx].ptr |= ptr & PAGE_MASK; + update_queue[idx].val = MMUEXT_INVLPG; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_pgd_pin(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = phys_to_machine(ptr); + update_queue[idx].ptr |= MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_PIN_L2_TABLE; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_pgd_unpin(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = phys_to_machine(ptr); + update_queue[idx].ptr |= MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_UNPIN_TABLE; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_pte_pin(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = phys_to_machine(ptr); + update_queue[idx].ptr |= MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_PIN_L1_TABLE; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_pte_unpin(unsigned long ptr) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = phys_to_machine(ptr); + update_queue[idx].ptr |= MMU_EXTENDED_COMMAND; + update_queue[idx].val = MMUEXT_UNPIN_TABLE; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_set_ldt(unsigned long ptr, unsigned long len) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = MMU_EXTENDED_COMMAND | ptr; + update_queue[idx].val = MMUEXT_SET_LDT | (len << MMUEXT_CMD_SHIFT); + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + +void xen_machphys_update(unsigned long mfn, unsigned long pfn) +{ + unsigned long flags; + spin_lock_irqsave(&update_lock, flags); + update_queue[idx].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE; + update_queue[idx].val = pfn; + increment_index_and_flush(); + spin_unlock_irqrestore(&update_lock, flags); +} + #ifdef CONFIG_XEN_PHYSDEV_ACCESS unsigned long allocate_empty_lowmem_region(unsigned long pages) diff --git a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/init.c b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/init.c index 12da77b2cb..0b2706b055 100644 --- a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/init.c +++ b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/init.c @@ -125,6 +125,7 @@ static void __init page_table_range_init (unsigned long start, unsigned long end } } +/* NOTE: caller must call flush_page_update_queue() */ void __init wrprotect_bootpt(pgd_t *pgd, void *page, int set) { pmd_t *pmd; @@ -141,6 +142,7 @@ void __init wrprotect_bootpt(pgd_t *pgd, void *page, int set) pte_val_ma(*pte) | _PAGE_RW); } +/* NOTE: caller must call flush_page_update_queue() */ static void __init protect_bootpt_entries(pgd_t *spgd, pgd_t *dpgd, int set, int pmdupdate, int pmdset) { @@ -163,7 +165,6 @@ static void __init protect_bootpt_entries(pgd_t *spgd, pgd_t *dpgd, int set, } } } - flush_page_update_queue(); } static inline int is_kernel_text(unsigned long addr) @@ -575,14 +576,13 @@ void __init paging_init(void) protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir, 1, 1, 1); queue_pgd_pin(__pa(swapper_pg_dir)); - flush_page_update_queue(); load_cr3(swapper_pg_dir); - __flush_tlb_all(); + __flush_tlb_all(); /* implicit flush */ queue_pgd_unpin(__pa(start_info.pt_base)); - flush_page_update_queue(); protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir, 0, 1, 0); wrprotect_bootpt((pgd_t *)start_info.pt_base, swapper_pg_dir, 0); + flush_page_update_queue(); #ifdef CONFIG_X86_PAE /* diff --git a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/pgtable.c b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/pgtable.c index aaf7339661..b6182d9961 100644 --- a/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/pgtable.c +++ b/linux-2.6.8.1-xen-sparse/arch/xen/i386/mm/pgtable.c @@ -180,6 +180,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) if (pte) { clear_page(pte); __make_page_readonly(pte); + xen_flush_page_update_queue(); } return pte; } @@ -204,7 +205,10 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) #ifdef CONFIG_HIGHPTE if (pte < highmem_start_page) #endif - __make_page_readonly(phys_to_virt(page_to_pseudophys(pte))); + { + __make_page_readonly(phys_to_virt(page_to_pseudophys(pte))); + flush_page_update_queue(); + } } return pte; } diff --git a/linux-2.6.8.1-xen-sparse/drivers/xen/blkfront/blkfront.c b/linux-2.6.8.1-xen-sparse/drivers/xen/blkfront/blkfront.c index e414259913..eb8fc0f9f0 100644 --- a/linux-2.6.8.1-xen-sparse/drivers/xen/blkfront/blkfront.c +++ b/linux-2.6.8.1-xen-sparse/drivers/xen/blkfront/blkfront.c @@ -1263,7 +1263,7 @@ void blkif_completion(blkif_request_t *req) { unsigned long pfn = req->frame_and_sects[i] >> PAGE_SHIFT; unsigned long mfn = phys_to_machine_mapping[pfn]; - queue_machphys_update(mfn, pfn); + xen_machphys_update(mfn, pfn); } break; } diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/mmu_context.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/mmu_context.h index cea0fd614c..223c7192c6 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/mmu_context.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/mmu_context.h @@ -39,7 +39,7 @@ static inline void switch_mm(struct mm_struct *prev, cpu_set(cpu, next->cpu_vm_mask); /* Re-load page tables */ - load_cr3_noflush(next->pgd); + load_cr3(next->pgd); /* * load the LDT, if the LDT is different: @@ -56,7 +56,7 @@ static inline void switch_mm(struct mm_struct *prev, /* We were in lazy tlb mode and leave_mm disabled * tlb flush IPI delivery. We must reload %cr3. */ - load_cr3_noflush(next->pgd); + load_cr3(next->pgd); load_LDT_nolock(&next->context, cpu); } } diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgalloc.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgalloc.h index f9b4709e50..0ba1e5a542 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgalloc.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgalloc.h @@ -33,6 +33,7 @@ static inline void pte_free_kernel(pte_t *pte) { free_page((unsigned long)pte); __make_page_writable(pte); + flush_page_update_queue(); } static inline void pte_free(struct page *pte) @@ -43,6 +44,7 @@ static inline void pte_free(struct page *pte) { __make_page_writable(phys_to_virt(page_to_pseudophys(pte))); __free_page(pte); + flush_page_update_queue(); } } diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h index c3d554ca70..1defe99a79 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable-2level.h @@ -27,14 +27,14 @@ static inline int pgd_present(pgd_t pgd) { return 1; } #define set_pte(pteptr, pteval) (*(pteptr) = pteval) #define set_pte_atomic(pteptr, pteval) (*(pteptr) = pteval) #else -#define set_pte(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low) -#define set_pte_atomic(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low) +#define set_pte(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low) +#define set_pte_atomic(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low) #endif /* * (pmds are folded into pgds so this doesn't get actually called, * but the define is needed for a generic inline function.) */ -#define set_pmd(pmdptr, pmdval) queue_l2_entry_update((pmdptr), (pmdval).pmd) +#define set_pmd(pmdptr, pmdval) xen_l2_entry_update((pmdptr), (pmdval).pmd) #define set_pgd(pgdptr, pgdval) ((void)0) #define pgd_page(pgd) \ diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable.h index a78420491c..6721e85aaf 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/pgtable.h @@ -252,7 +252,7 @@ static inline int ptep_test_and_clear_dirty(pte_t *ptep) pte_t pte = *ptep; int ret = pte_dirty(pte); if (ret) - queue_l1_entry_update(ptep, pte_mkclean(pte).pte_low); + xen_l1_entry_update(ptep, pte_mkclean(pte).pte_low); return ret; } @@ -261,7 +261,7 @@ static inline int ptep_test_and_clear_young(pte_t *ptep) pte_t pte = *ptep; int ret = pte_young(pte); if (ret) - queue_l1_entry_update(ptep, pte_mkold(pte).pte_low); + xen_l1_entry_update(ptep, pte_mkold(pte).pte_low); return ret; } @@ -275,7 +275,7 @@ static inline void ptep_mkdirty(pte_t *ptep) { pte_t pte = *ptep; if (!pte_dirty(pte)) - queue_l1_entry_update(ptep, pte_mkdirty(pte).pte_low); + xen_l1_entry_update(ptep, pte_mkdirty(pte).pte_low); } /* @@ -318,7 +318,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) pmd_t p = *(xp); \ set_pmd(xp, __pmd(0)); \ __make_page_writable((void *)pmd_page_kernel(p)); \ - /* XXXcl queue */ \ + xen_flush_page_update_queue(); \ } while (0) #ifndef CONFIG_DISCONTIGMEM @@ -418,12 +418,13 @@ extern pte_t *lookup_address(unsigned long address); #define update_mmu_cache(vma,address,pte) do { } while (0) #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -#if 1 +#if 0 #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ do { \ if (__dirty) { \ queue_l1_entry_update((__ptep), (__entry).pte_low); \ - flush_tlb_page(__vma, __address); \ + flush_tlb_page(__vma, __address); \ + xen_flush_page_update_queue(); \ } \ } while (0) #else @@ -434,8 +435,7 @@ extern pte_t *lookup_address(unsigned long address); xen_flush_page_update_queue(); \ HYPERVISOR_update_va_mapping(address>>PAGE_SHIFT, entry, UVMF_INVLPG); \ } else { \ - queue_l1_entry_update((__ptep), (__entry).pte_low); \ - xen_flush_page_update_queue(); \ + xen_l1_entry_update((__ptep), (__entry).pte_low); \ } \ } \ } while (0) @@ -449,6 +449,7 @@ extern pte_t *lookup_address(unsigned long address); #define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) +/* NOTE: make_page* callers must call flush_page_update_queue() */ static inline void __make_page_readonly(void *va) { pgd_t *pgd = pgd_offset_k((unsigned long)va); @@ -474,7 +475,6 @@ static inline void make_page_readonly(void *va) if ( (unsigned long)va >= VMALLOC_START ) __make_page_readonly(machine_to_virt( *(unsigned long *)pte&PAGE_MASK)); - /* XXXcl queue */ } static inline void make_page_writable(void *va) @@ -486,7 +486,6 @@ static inline void make_page_writable(void *va) if ( (unsigned long)va >= VMALLOC_START ) __make_page_writable(machine_to_virt( *(unsigned long *)pte&PAGE_MASK)); - /* XXXcl queue */ } static inline void make_pages_readonly(void *va, unsigned int nr) @@ -496,7 +495,6 @@ static inline void make_pages_readonly(void *va, unsigned int nr) make_page_readonly(va); va = (void *)((unsigned long)va + PAGE_SIZE); } - /* XXXcl queue */ } static inline void make_pages_writable(void *va, unsigned int nr) @@ -506,7 +504,6 @@ static inline void make_pages_writable(void *va, unsigned int nr) make_page_writable(va); va = (void *)((unsigned long)va + PAGE_SIZE); } - /* XXXcl queue */ } static inline unsigned long arbitrary_virt_to_phys(void *va) diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/processor.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/processor.h index 7b4ca3fdb9..b8497743bb 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/processor.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/processor.h @@ -181,12 +181,6 @@ static inline unsigned int cpuid_edx(unsigned int op) } #define load_cr3(pgdir) do { \ - queue_pt_switch(__pa(pgdir)); \ - flush_page_update_queue(); \ - cur_pgd = pgdir; /* XXXsmp */ \ -} while (/* CONSTCOND */0) - -#define load_cr3_noflush(pgdir) do { \ queue_pt_switch(__pa(pgdir)); \ cur_pgd = pgdir; /* XXXsmp */ \ } while (/* CONSTCOND */0) diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/tlbflush.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/tlbflush.h index 59fa36f82d..bb1c124907 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/tlbflush.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/asm-i386/tlbflush.h @@ -6,8 +6,7 @@ #include #define __flush_tlb() do { \ - queue_tlb_flush(); \ - xen_flush_page_update_queue(); \ + xen_tlb_flush(); \ } while (/*CONSTCOND*/0) /* @@ -16,8 +15,7 @@ */ #define __flush_tlb_global() \ do { \ - queue_tlb_flush(); \ - xen_flush_page_update_queue(); \ + xen_tlb_flush(); \ } while (0) extern unsigned long pgkern_mask; @@ -33,8 +31,7 @@ extern unsigned long pgkern_mask; #define cpu_has_invlpg (boot_cpu_data.x86 > 3) #define __flush_tlb_single(addr) do { \ - queue_invlpg(addr); \ - xen_flush_page_update_queue(); \ + xen_invlpg(addr); \ } while (/* CONSTCOND */0) # define __flush_tlb_one(addr) __flush_tlb_single(addr) diff --git a/linux-2.6.8.1-xen-sparse/include/asm-xen/hypervisor.h b/linux-2.6.8.1-xen-sparse/include/asm-xen/hypervisor.h index a3b180ecb8..4198263d14 100644 --- a/linux-2.6.8.1-xen-sparse/include/asm-xen/hypervisor.h +++ b/linux-2.6.8.1-xen-sparse/include/asm-xen/hypervisor.h @@ -58,6 +58,17 @@ void queue_pte_pin(unsigned long ptr); void queue_pte_unpin(unsigned long ptr); void queue_set_ldt(unsigned long ptr, unsigned long bytes); void queue_machphys_update(unsigned long mfn, unsigned long pfn); +void xen_l1_entry_update(pte_t *ptr, unsigned long val); +void xen_l2_entry_update(pmd_t *ptr, unsigned long val); +void xen_pt_switch(unsigned long ptr); +void xen_tlb_flush(void); +void xen_invlpg(unsigned long ptr); +void xen_pgd_pin(unsigned long ptr); +void xen_pgd_unpin(unsigned long ptr); +void xen_pte_pin(unsigned long ptr); +void xen_pte_unpin(unsigned long ptr); +void xen_set_ldt(unsigned long ptr, unsigned long bytes); +void xen_machphys_update(unsigned long mfn, unsigned long pfn); #define MMU_UPDATE_DEBUG 0 #if MMU_UPDATE_DEBUG > 0 -- 2.30.2